#masked language modeling

def prune_transformer(model, amount=0.3):
            for name, module in model.named_modules():
                if isinstance(module, torch.nn.Linear):
                    prune.l1_unstructured(module, name='weight', amount=amount)
                    prune.remove(module, 'weight')
            return model

        # Efficient Multilingual NMT
        class LightTranslator(nn.Module):
            def __init__(self, vocab_size, d_model=512):
                super().__init__()
                self.encoder = TransformerEncoder(d_model)
                self.decoder = TransformerDecoder(d_model)

            def forward(self, src, tgt):
                memory = self.encoder(src)
                return self.decoder(tgt, memory)

AraSSM: A Bidirectional State-Space Encoder for Arabic Masked Language Modeling

2026-08-08Ahmed Amine Aliane, Hassina Aliane, Nasredine SemmararXiv preprint

[NLP] [State Space Models] [Mamba] [Masked Language Modeling] [Arabic]

A bidirectional Mamba/SSM encoder pretrained from scratch for Arabic via masked language modeling, reaching near-parity with base-sized Transformer encoders on consumer-grade hardware.